/*
 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
 *
 * SPDX-License-Identifier: GPL-2.0-only
 */

#include <config.h>
#include <arch/machine.h>
#include <arch/kernel/boot_sys.h>
#include <arch/kernel/smp_sys.h>
#include <smp/lock.h>

#ifdef ENABLE_SMP_SUPPORT
/*Z 下一个启动cpu索引，每个cpu启动后都增加该值 */
/* Index of next AP to boot, BSP has index zero */
BOOT_DATA VISIBLE
volatile word_t smp_aps_index = 1;

#ifdef CONFIG_USE_LOGICAL_IDS
/*Z 更新cpu逻辑id和cluster id位图 */
BOOT_CODE static void update_logical_id_mappings(void)
{   /*Z 获取逻辑cpu id */
    cpu_mapping.index_to_logical_id[getCurrentCPUIndex()] = apic_get_logical_id();
    /*Z 对同一cluster中的cpu，更新cluster id位图 */
    for (int i = 0; i < smp_aps_index; i++) {
        if (apic_get_cluster(cpu_mapping.index_to_logical_id[getCurrentCPUIndex()]) ==
            apic_get_cluster(cpu_mapping.index_to_logical_id[i])) {

            cpu_mapping.other_indexes_in_cluster[getCurrentCPUIndex()] |= BIT(i);
            cpu_mapping.other_indexes_in_cluster[i] |= BIT(getCurrentCPUIndex());
        }
    }
}
#endif /* CONFIG_USE_LOGICAL_IDS */
/*Z 向指定cpu发送初始化和启动IPI，该cpu开始执行指定物理地址启动代码 */
BOOT_CODE static void start_cpu(cpu_id_t cpu_id, paddr_t boot_fun_paddr)
{
    /* memory fence needed before starting the other CPU */
    x86_mfence();

    /* starting the other CPU */
    apic_send_init_ipi(cpu_id);/*Z 向指定cpu发送初始化IPI */
    apic_send_startup_ipi(cpu_id, boot_fun_paddr);/*Z 向指定cpu发送启动IPI，该cpu开始执行物理地址startup_addr处启动代码 */
}
/*Z 根据ACPI信息，建立cpu索引到cpu id的映射，并启动其它核 */
BOOT_CODE void start_boot_aps(void)
{
    /* update cpu mapping for BSP, cpus[0] is always assumed to be BSP */
    cpu_mapping.index_to_cpu_id[getCurrentCPUIndex()] = boot_state.cpus[0];
#ifdef CONFIG_USE_LOGICAL_IDS
    cpu_mapping.index_to_logical_id[getCurrentCPUIndex()] = apic_get_logical_id();
#endif /* CONFIG_USE_LOGICAL_IDS */
    /*Z 依次启动所有核 */
    /* startup APs one at a time as we use shared kernel boot stack */
    while (smp_aps_index < boot_state.num_cpus) {
        word_t current_ap_index = smp_aps_index;

        printf("Starting node #%lu with APIC ID %lu \n",
               current_ap_index, boot_state.cpus[current_ap_index]);
        /*Z 向指定cpu发送初始化和启动IPI（依据的是MP协议），该cpu开始执行指定物理地址启动代码 */
        /* update cpu mapping for APs, store APIC ID of the next booting AP
         * as APIC ID are not continoius e.g. 0,2,1,3 for 4 cores with hyperthreading
         * we need to store a mapping to translate the index to real APIC ID */
        cpu_mapping.index_to_cpu_id[current_ap_index] = boot_state.cpus[current_ap_index];
        start_cpu(boot_state.cpus[current_ap_index], BOOT_NODE_PADDR);
        /*Z 等待该核启动完毕（该核清中断，进入HALT状态） */
        /* wait for current AP to boot up */
        while (smp_aps_index == current_ap_index);
    }
}
/*Z 将多核引导代码拷贝到低端640K内存内，mem_lower指明低端内存数量 */
BOOT_CODE bool_t copy_boot_code_aps(uint32_t mem_lower)
{
    assert(boot_cpu_end - boot_cpu_start < 0x400);

    /* Ensure that our boot code fits in the memory hole we want to use, and check this region
     * is free according to multiboot. As boot_cpu_end and boot_cpu_start are link time
     * symbols (and not compile time) this cannot be a compile time check */
    word_t boot_size = (word_t)(boot_cpu_end - boot_cpu_start);
    word_t boot_node_top = BOOT_NODE_PADDR + boot_size;
    word_t mem_lower_bytes = mem_lower << 10;   /*Z KB -> B */
    if (boot_node_top > BOOT_NODE_MAX_PADDR) {
        printf("AP boot code does not fit in chosen memory hole. Can be at most %lu, is %lu\n",
               (word_t)(BOOT_NODE_MAX_PADDR - BOOT_NODE_PADDR), boot_size);
        return false;
    }
    if (mem_lower_bytes < boot_node_top) {
        printf("Need lower physical memory up to %lu to be free. Multiboot reports only up to %lu\n",
               boot_node_top, mem_lower_bytes);
        return false;
    }

    /* copy CPU bootup code to lower memory */
    memcpy((void *)BOOT_NODE_PADDR, boot_cpu_start, boot_size);
    return true;
}
/*Z 其它核启动代码：初始化CPU，建立内核关键数据 */
static BOOT_CODE bool_t try_boot_node(void)
{   /*Z 将物理地址addr作为一级页表，和用户进程标识pcid一起写入CR3寄存器，从而确定一个虚拟地址空间 */
    setCurrentVSpaceRoot(kpptr_to_paddr(X86_KERNEL_VSPACE_ROOT), 0);
    /* Sync up the compilers view of the world here to force the PD to actually
     * be set *right now* instead of delayed */
    asm volatile("" ::: "memory");
    /*Z x86的启动代码部分：初始化TSS段、GDT、IDT表，加载内存管理寄存器GDTR、IDTR、LDTR、TR，
初始化系统调用指令、PAT页缓存属性、分支预测、APIC等MSR寄存器，设置CR0、CR4保护位、定时器、FPU，
可配置地设置VMX虚拟机支持，进入VMM管理模式 */
    /* initialise the CPU, make sure legacy interrupts are disabled */
    if (!init_cpu(1)) {
        return false;
    }

#ifdef CONFIG_USE_LOGICAL_IDS
    update_logical_id_mappings();/*Z 更新cpu逻辑id和cluster id位图 */
#endif /* CONFIG_USE_LOGICAL_IDS */
    return true;
}
/*Z 其它核启动代码第二部分 */
/* This is the entry function for APs. However, it is not a BOOT_CODE as
 * there is a race between exiting this function and root task running on
 * node #0 to possibly reallocate this memory */
VISIBLE void boot_node(void)
{
    bool_t result;
    /*Z 初始化CPU，建立内核关键数据 */
    mode_init_tls(smp_aps_index);
    result = try_boot_node();

    if (!result) {
        fail("boot_node failed for some reason :(\n");
    }

    smp_aps_index++;
    /*Z 到这里向引导核发出启动完毕的信号。也就是并行开始了 */
    /* grab BKL before leaving the kernel */
    NODE_LOCK_SYS;/*Z 获取内核锁 */
    /*Z 初始化系统运行状态 */
    init_core_state(SchedulerAction_ChooseNewThread);
    ARCH_NODE_STATE(x86KScurInterrupt) = int_invalid;
    ARCH_NODE_STATE(x86KSPendingInterrupt) = int_invalid;

    schedule();
    activateThread();
}/*Z 这个函数返回后，ret指令弹出的是restore_user_context()地址，相应的会释放内核锁 */

#endif /* ENABLE_SMP_SUPPORT */
